#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
sys.path.append("/work/opt/analytic/my_python/lib")
from lib.format_time import *
from lib.HandleMysql import *
from lib.switch_code import *
from lib.email_helper import *
from jpype import *
from lib.load_data import *
import urllib
import glob
import sys
import re
import os
sys.setdefaultencoding('utf-8')

class Service58(object):
    def __init__(self):
        self.jars = ':'.join(glob.glob('/work/opt/analytic/my_python/java_lib/imc/*.jar'))
        self.jvm_path = '/opt/soft/java/jre/lib/amd64/server/libjvm.so'
        startJVM(self.jvm_path,"-ea","-Djava.class.path=%s" % self.jars)
        self.SCFInit =JClass("com.bj58.spat.scf.client.SCFInit")
        self.SCFInit.init('/work/opt/analytic/my_python/config/scf.config')
        
        URL= "tcp://imc/InfoProviderWithIndex"            
        self.ProxyFactory =JClass("com.bj58.spat.scf.client.proxy.builder.ProxyFactory")
        #self.IProductService =JClass("com.bj58.sfft.imc.cntity.Info")
        self.Info =JClass("com.bj58.sfft.imc.contract.IInfoService")
        self.Integer =JClass("java.lang.Integer")
        self.proxy = self.ProxyFactory.create(self.Info,URL)
        #self.out = JClass("com.bj58.spat.scf.server.contract.entity.Out")
        #self.out_total = self.out{}
        print "ok"
        
    def get_info(self,user_Id): #
        self.user_Id = user_Id
        self.query = "state=1&userid=%s"%user_Id
        self.info_count = self.proxy.GetInfoCount(self.query)
        
        return self.info_count
        
    def shut_down(self):
        shutdownJVM()
    
    
#mysql - stats
def set_open_conn():    
    conn = HandleMysql('10.5.20.16','58885','ec_stat','L#c8UM5RGh5C','dbwww58com_58_stats')
    conn.connect()
    return conn
#mysql - stats
def set_dict_conn():    
    conn = HandleMysql('10.5.20.16','58885','ec_stat','L#c8UM5RGh5C','dbwww58com_base_dicts')
    conn.connect()
    return conn

#logic
def execute( s58,date ):
    conn = set_open_conn();
    month = get_date_info(date,'%Y%m')
    stat_table = "user_post_stats_all%s"%month
    line_list = []
    line_user_list = []
    line_user_tmp_list = []
    try:
        pass
        create_post_table(conn,stat_table)
    except Exception,e:
        print 'create table error:',e
    
    query = "select distinct user_id,ent_name from order_info where pay_amount > 0  and order_state in (35,45,50)"
    #query = "select user_id,ent_name from user_post_stats_all where info_count=0"
    items = conn.get_results(query)
    for item in items:
        user_id = item[0]
        user_name = item[1]
        #print user_id
        cnt = -1
        query_in = "select cur_info_count from user_post_base where user_id=%s;"%user_id
        items_in = conn.get_results(query_in)
        if len(items_in)>0:
            cnt = items_in[0][0]
        #print cnt
        
        info_count = 0
        post_count = 0
        user_post_count = 0
        cs_post_count = 0
        
        if cnt == -1 or cnt == None :               
            #print user_id
            try:
                info_count= s58.get_info(user_id)
                #print info_count ,post_count
                param_user = [user_id,info_count,info_count]
                line_user_list.append('\t'.join(map(lambda i : str(i), param_user)))
            except Exception,e:
                print user_id,e         
        else:
            info_count = cnt
            
        #print info_count
        query_in = "select ifnull(info_count,0),ifnull(cs_post,0) from user_post_hp where state=1 and user_id=%s and stat_date='%s';"%(user_id,date)
        items_in = conn.get_results(query_in)
        for it in items_in:
            i_count = it[0]
            cs_post = it[1]
            
            if cs_post == 0:
                user_post_count = i_count
            else:
                cs_post_count = i_count
        
        post_count = user_post_count + cs_post_count
        
        #print post_count
        if info_count == None:
            info_count = 0
            
        info_count =  info_count + post_count
        param = [date,user_id,user_name,info_count,post_count,cs_post_count,user_post_count]
        line_list.append('\t'.join(map(lambda i : str(i), param)))
        if cnt > -1 or cnt == None:
            param = [user_id,info_count]
            #print cnt,param
            line_user_tmp_list.append('\t'.join(map(lambda i : str(i), param)))
    #return   
    cont = '\n'.join(line_list)
    path = "/work3/opt/analytic/lh_tmp_data/user_info/"
    file_name = "user_post_stats_%s.txt"%date.replace('-',"")
    writeTextToFile(path,file_name,"w",cont)
    load_data(conn,path+file_name,stat_table,'\t')
    
    cont_user = '\n'.join(line_user_list)
    user_file_name = "user_post_newuser_stats_%s.txt"%date.replace('-',"")
    writeTextToFile(path,user_file_name,"w",cont_user)
    
    load_data(conn,path+user_file_name,"user_post_base",'\t')
    
    cont_user_tmp = '\n'.join(line_user_tmp_list)
    user_tmp_file_name = "user_post_tmp_stats_%s.txt"%date.replace('-',"")
    writeTextToFile(path,user_tmp_file_name,"w",cont_user_tmp)
    query = "truncate table user_post_tmp;"
    conn.execute_sql(query)
    
    #print cont_user_tmp
    load_data(conn,path+user_tmp_file_name,"user_post_tmp",'\t')
    
    query = "update user_post_base a left join user_post_tmp b on a.user_id=b.user_id set a.cur_info_count=b.info_count;"
    conn.execute_sql(query)
    
    conn.close()


def create_post_table(conn,table_name):
    q = '''\
        CREATE TABLE if not exists  `%s` (
        `stat_date` date NOT NULL DEFAULT '0000-00-00' COMMENT '统计日期',
        `user_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '用户ID',
        `ent_name` varchar(128) DEFAULT NULL COMMENT '企业名称',
        `info_count` int(11) DEFAULT NULL COMMENT '有效发贴数',
        `post_count` int(11) DEFAULT NULL COMMENT '当日发贴数',
        `cs_post_count` int(11) DEFAULT '0' COMMENT '客服发贴',
        `user_post_count` int(11) DEFAULT '0' COMMENT '用户发贴数',
        PRIMARY KEY (`user_id`,`stat_date`)
      ) ENGINE=MyISAM DEFAULT CHARSET=utf8;

    ''' % table_name
    
    conn.execute_sql(q)
    
def get_user_post(date):
    #print date

    date_str = date.replace('-',"")    
    url_str = "http://10.5.12.160:50075/streamFile/dsap/resultdata/ec_data/EC_PostJob/%s/part-r-00000"%date_str
    file_path = "/work3/opt/analytic/lh_tmp_data/user_post/info_%s.txt"%date_str
    
    #print url_str,file_path    
    urllib.urlretrieve(url_str, file_path)
    conn = set_open_conn();
    line_list = []    
    
    stat_table = "user_post_hp"
    #process file
    f = open(file_path)
    for line in f.readlines():
        #user9369905819143_state1_cspost_1_count:2
        #print line
        arr = line.replace('\n','').split(':')
        key = arr[0]
        info_count = arr[1]
        
        key = key.replace('_count','')
        key = key.replace('user','')
        key = key.replace('state','')
        key = key.replace('_cspost','')
        tmp = key.split('_')
        
        user_id = "0"
        state = "0"
        cs_post = "0"
        try:
            user_id = tmp[0]
            state = tmp[1]
            cs_post = tmp[2]
        except Exception:
            continue
        
        #if cs_post == "1":
            #print line
        param = [date,user_id,state,info_count,cs_post]
        line_list.append('\t'.join(map(lambda i : str(i), param)))
        
    cont = '\n'.join(line_list)
    path = "/work3/opt/analytic/lh_tmp_data/user_info/"
    file_name = "haddop_user_post_stats_%s.txt"%date.replace('-',"")
    writeTextToFile(path,file_name,"w",cont)
    
    load_data(conn,path+file_name,stat_table,'\t')
    conn.close()

def post_info_monitor(date ):
    
    conn = set_open_conn();
    month = date.replace('-',"")[0:6]
    #date = date[0:6]
    m_table = "user_post_%s"%month
    #print m_table
    
    query = "select count(1) from %s where stat_date='%s'"%(m_table,date)
    items = conn.get_results(query)
    cnt = items[0][0]
    
    if cnt == 0:
        mail_to="datamining@58.com"
        sub = "error:table %s has no data at %s"%(m_table,date)
        cont = "fyi"
        send_mail(mail_to,sub,cont)
    
#logic
def user_post_distributed_stats( date ):
    conn = set_open_conn();
    month = get_date_info(date,'%Y%m')
    stat_table = "user_post_distributed_stats%s"%month
    line_list = []
    
    try:
        pass
        create_distributed_table(conn,stat_table)
    except Exception,e:
        print 'create table create_distributed_table error:',e
    
    if len(channel_dict) == 0 :
        get_channel_dict()
        
    date_str = date.replace('-',"")
    for i in range(8):
        
        url_str = "http://10.5.12.160:50075/streamFile/dsap/resultdata/ec_data/EC_UserPostJob/channel/%s/part-r-0000%s"%(date_str,i)
        file_path = "/work3/opt/analytic/lh_tmp_data/user_post/info_distributed_%s_%s.txt"%(date_str,i)
        
        #print url_str,file_path    
        urllib.urlretrieve(url_str, file_path)
        conn = set_open_conn();
    
        #process file
        f = open(file_path)
        for line in f.readlines():
            #cate0,1,10_local10015_user10877808026119_source0_uptype0_icount:1
            #print line
            arr = line.replace('\n','').split(':')
            key = arr[0]
            info_count = arr[1]
            
            key = key.replace('_icount','')
            key = key.replace('cate','')
            key = key.replace('local','')
            key = key.replace('user','')
            key = key.replace('source','')
            key = key.replace('uptype','')
            
            tmp = key.split('_')
            #print tmp
            cate = ""
            local = ""
            user_id = "0"
            source = "0"
            uptype = "0"
            try:
                cate = tmp[0]
                local = tmp[1]
                user_id = tmp[2]
                source = tmp[3]
                uptype = tmp[4]
            except Exception:
                print line
                continue
            
            channel_id = 0
            channel_name = ""
            cate_arr = cate.split(",")
            cate_id = cate_arr[len(cate_arr)-1]
            if channel_dict.has_key(cate_id):
                channel = channel_dict[cate_id]
                ch_arr = channel.split("_")
                channel_id = ch_arr[0]
                channel_name = ch_arr[1]
                
            #print "%s,%s"%(source,uptype)
            if int(source) > 6:
                continue
            
            param = [date,user_id,uptype,source,info_count,channel_id,channel_name,cate,local]
            #print param
            line_list.append('\t'.join(map(lambda i : str(i), param)))
    
        remove_file(file_path)  
    cont = '\n'.join(line_list)
    path = "/work3/opt/analytic/lh_tmp_data/user_info/"
    file_name = "haddop_user_post_distributed_stats_%s.txt"%date.replace('-',"")
    writeTextToFile(path,file_name,"w",cont)
    
    load_data(conn,path+file_name,stat_table,'\t')
    
    remove_file(path+file_name)
    conn.close()
    
   
channel_dict = {}
def get_channel_dict():
    conn = set_dict_conn()
    q = "select distinct cate_id,channel_id,channel_name from cate_channel"
    items = conn.get_results(q)
    for item in items:
        cate_id = item[0]
        channel_id = item[1]
        channel_name = item[2]
        
        val = "%s_%s"%(channel_id,channel_name)
        if not channel_dict.has_key(str(cate_id)):
            channel_dict[str(cate_id)] = val
    conn.close()

def create_distributed_table(conn,table_name):
    q = '''\
        CREATE TABLE `%s` (
        `stat_date` date NOT NULL DEFAULT '0000-00-00',
        `user_id` bigint(20) NOT NULL DEFAULT '0',
        `up_type` tinyint(4) DEFAULT '0' COMMENT '帖子更新状态，0为新发帖子，1修改，2刷新',
        `is_vip` tinyint(4) DEFAULT '0' COMMENT '来源，0为个人，6为VIP',
        `info_count` int(11) DEFAULT NULL,
        `channel_id` int(11) DEFAULT NULL,
        `channel_name` varchar(32) DEFAULT NULL,
        `cates` varchar(48) DEFAULT NULL,
        `locals` varchar(48) DEFAULT NULL
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

    ''' % table_name
    
    conn.execute_sql(q)
def main():

    #date = '2012-05-28'
   #execute(date)
   inp = 0
   if len(sys.argv)==1:
        start_date=get_yesterday()
        end_date=start_date
   elif len(sys.argv)==2:
        inp = sys.argv[1]
        start_date=format_date(sys.argv[1])
        end_date=start_date
   elif len(sys.argv)==3:
        start_date=format_date(sys.argv[1])
        end_date=format_date(sys.argv[2])
   if inp == "1":
        #start_date = "2013-03-07"
        start_date=get_yesterday()
        post_info_monitor(start_date)
   else:
        s58=Service58()
        stat_date = start_date
        #conn = set_open_conn();
        while stat_date>=start_date and stat_date<=end_date:
             print "The program of the date of %s is beginning at %s " %(stat_date,get_now())
             #get hadoop data
             get_user_post(stat_date)
             
             #process data
             execute( s58,stat_date)
            
            #
             user_post_distributed_stats(stat_date)
             #stat_table = "user_post_stats_all201306"
             #path = "/work3/opt/analytic/lh_tmp_data/user_info/"
             #file_name = "user_post_stats_%s.txt"%stat_date.replace('-',"")
             #load_data(conn,path+file_name,stat_table,'\t')
             print "The program of the date of %s is end at %s " %(stat_date,get_now())
             stat_date=var_date(stat_date)

if __name__=='__main__':
    main()
